#!/usr/bin/env python
# -*- coding:utf-8 -*-
# author:sirian
# datetime:2018/11/6 21:39
# software: PyCharm

'''
利用正则表达式，下载页面段子内容

'''

import urllib2
import re


class spider:
    def __init__(self):
        self.control_tag = True
        self.page = 60
        self.filename = raw_input("要写入的文件名:")


    #下载方法
    def upload_spider(self):
        ua = {
            "Connection": "keep-alive",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36",
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
            "Accept-Language": "zh-CN,zh;q=0.9"
        }
        url = "http://jandan.net/duan/page-"+str(self.page)
        print(url)
        #创建一个HTTPHandler对象处理HTTP请求
        http_handler = urllib2.HTTPHandler()

        #创建一个opener，利用urllib2的build_opener()方法
        opener = urllib2.build_opener(http_handler)

        request = urllib2.Request(url, headers=ua)

        response = opener.open(request)

        html = response.read()

        pattern = re.compile('<div class="text"><span class="righttext">.*?<p>(.*?)</p>.*?</div>', re.S)

        content = pattern.findall(html)

        #print(content)

        self.deal_spider(content)


    #处理下载文件，去除多余内容
    def deal_spider(self, content):

        for text in content:
            text = text.replace("<br />","")
            text = text
            print(text)

            self.write_spider(text)



    #将段子写入到文件，保存
    def write_spider(self, text):

        with open(self.filename, 'a') as f:
            f.write(text)


    #控制爬虫
    def control_spider(self):

        while self.control_tag:
            self.upload_spider()
            continue_spider = raw_input("if continue please enter or don`t continue (quite):")
            self.page = self.page - 1
            if continue_spider == 'quite':
                self.control_tag = False
        print("结束下载任务！")






if __name__ == "__main__":
    dz_spider = spider()
    dz_spider.control_spider()
